{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/py38/lib/python3.8/site-packages/sklearn/datasets/_openml.py:932: FutureWarning: The default value of `parser` will change from `'liac-arff'` to `'auto'` in 1.4. You can set `parser='auto'` to silence this warning. Therefore, an `ImportError` will be raised from 1.4 if the dataset is dense and pandas is not installed. Note that the pandas parser may return different data types. See the Notes Section in fetch_openml's API doc for details.\n",
      "  warn(\n",
      "/tmp/ipykernel_3969898/3404423563.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data[col] = le.fit_transform(data[col])\n",
      "/tmp/ipykernel_3969898/3404423563.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data[col] = le.fit_transform(data[col])\n",
      "/tmp/ipykernel_3969898/3404423563.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data[col] = le.fit_transform(data[col])\n",
      "/tmp/ipykernel_3969898/3404423563.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data[col] = le.fit_transform(data[col])\n",
      "/tmp/ipykernel_3969898/3404423563.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data[col] = le.fit_transform(data[col])\n",
      "/tmp/ipykernel_3969898/3404423563.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data[col] = le.fit_transform(data[col])\n",
      "/tmp/ipykernel_3969898/3404423563.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data[col] = le.fit_transform(data[col])\n",
      "/tmp/ipykernel_3969898/3404423563.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data[col] = le.fit_transform(data[col])\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from sklearn.datasets import fetch_openml\n",
    "import numpy as np\n",
    "from types import SimpleNamespace\n",
    "from typing import Tuple, List\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder, MinMaxScaler\n",
    "    \n",
    "adult = fetch_openml(data_id = 1590, data_home='./data_cache')\n",
    "\n",
    "data = adult.data\n",
    "\n",
    "le = LabelEncoder()\n",
    "label = pd.Series(le.fit_transform(adult.target))\n",
    "\n",
    "\n",
    "category_cols = ['workclass', 'education', 'race', 'sex', \"marital-status\", \"occupation\", \"relationship\", \"native-country\"]\n",
    "continuous_cols = [x for x in data.columns if x not in category_cols]\n",
    "\n",
    "for col in category_cols:\n",
    "    data[col] = le.fit_transform(data[col])\n",
    "\n",
    "\n",
    "    \n",
    "    \n",
    "temp = None\n",
    "for col in category_cols:\n",
    "    oh_values = OneHotEncoder().fit_transform(data[col].values.reshape((-1, 1))).toarray()\n",
    "    new_cols = [col + \"-\" + str(i) for i in range(len(data[col].unique()))]\n",
    "    oh_values = pd.DataFrame(oh_values, columns = new_cols, dtype=np.int8, index=data.index)\n",
    "    if temp is None:\n",
    "        temp = oh_values\n",
    "    else:\n",
    "        temp = temp.merge(oh_values, left_index=True, right_index=True)\n",
    "\n",
    "data = data.merge(temp, left_index=True, right_index=True)\n",
    "data.drop(category_cols, inplace=True, axis=1)\n",
    "\n",
    "category_cols = temp.columns\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "data[continuous_cols] = scaler.fit_transform(data[continuous_cols])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/py38/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "num_categoricals = len(continuous_cols)\n",
    "num_continuous = len(continuous_cols)\n",
    "loss_fn = nn.CrossEntropyLoss\n",
    "metric =  \"accuracy_score\"\n",
    "metric_params = {}\n",
    "random_seed = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from misc.scorer import BaseScorer\n",
    "\n",
    "\n",
    "class AccuracyScorer(BaseScorer):\n",
    "    def __init__(self, metric: str) -> None:\n",
    "        super().__init__(metric)\n",
    "    \n",
    "    def __call__(self, y, y_hat) -> float:\n",
    "        return self.metric(y, y_hat.argmax(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pl_vime import PLVIME\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(data, label, train_size = 0.7, random_state=random_seed, stratify=label)\n",
    "\n",
    "X_train, X_unlabeled, y_train, _ = train_test_split(X_train, y_train, train_size = 0.1, random_state=random_seed, stratify=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytorch_lightning import Trainer\n",
    "from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from data_utils import *\n",
    "\n",
    "gpus = [1]\n",
    "n_jobs = 32\n",
    "max_epochs = 30\n",
    "batch_size = 512\n",
    "\n",
    "pretraining_patience = 10\n",
    "early_stopping_patience = 10\n",
    "\n",
    "batch_size = 512\n",
    "\n",
    "def fit_model(\n",
    "            model,\n",
    "            data_hparams\n",
    "    ):\n",
    "    \n",
    "    train_ds = VIMESelfDataset(X_train.append(X_unlabeled), data_hparams, continuous_cols, category_cols)\n",
    "    test_ds = VIMESelfDataset(X_valid, data_hparams, continuous_cols, category_cols)\n",
    "    \n",
    "    pl_datamodule = PLDataModule(train_ds, test_ds, batch_size=batch_size)\n",
    "\n",
    "    model.do_pretraining()\n",
    "\n",
    "    callbacks = [\n",
    "        EarlyStopping(\n",
    "            monitor= 'val_loss', \n",
    "            mode = 'min',\n",
    "            patience = pretraining_patience,\n",
    "            verbose = False\n",
    "        )\n",
    "    ]\n",
    "    pretraining_path = f'temporary_ckpt_data/pretraining'\n",
    "    checkpoint_callback = ModelCheckpoint(\n",
    "        monitor='val_loss',\n",
    "        dirpath=pretraining_path,\n",
    "        filename='pretraining-{epoch:02d}-{val_f1:.4f}',\n",
    "        save_top_k=1,\n",
    "        mode = 'min'\n",
    "    )\n",
    "\n",
    "    callbacks.append(checkpoint_callback)\n",
    "\n",
    "    trainer = Trainer(\n",
    "                    devices = gpus,\n",
    "                    accelerator = \"cuda\" if len(gpus) >= 1 else 'cpu',\n",
    "                    max_epochs = max_epochs,\n",
    "                    num_sanity_val_steps = 2,\n",
    "                    callbacks = callbacks,\n",
    "    )\n",
    "\n",
    "    trainer.fit(model, pl_datamodule)\n",
    "    \n",
    "    pretraining_path = checkpoint_callback.best_model_path\n",
    "\n",
    "    model = model.load_from_checkpoint(pretraining_path)\n",
    "\n",
    "    model.do_finetunning()\n",
    "    \n",
    "        \n",
    "    train_ds = VIMEClassificationDataset(X_train, y_train.values, data_hparams, X_unlabeled, continuous_cols, category_cols)\n",
    "    test_ds = VIMEClassificationDataset(X_valid, y_valid.values, data_hparams, None, continuous_cols, category_cols)\n",
    "\n",
    "    pl_datamodule = PLDataModule(train_ds, test_ds, batch_size = batch_size)\n",
    "        \n",
    "    callbacks = [\n",
    "        EarlyStopping(\n",
    "            monitor= 'val_' + metric, \n",
    "            mode = 'max',\n",
    "            patience = early_stopping_patience,\n",
    "            verbose = False\n",
    "        )\n",
    "    ]\n",
    "\n",
    "    checkpoint_path = None\n",
    "\n",
    "    checkpoint_path = f'temporary_ckpt_data/'\n",
    "    checkpoint_callback = ModelCheckpoint(\n",
    "        monitor='val_' + metric,\n",
    "        dirpath=checkpoint_path,\n",
    "        filename='{epoch:02d}-{val_f1:.4f}',\n",
    "        save_top_k=1,\n",
    "        mode = 'max'\n",
    "    )\n",
    "\n",
    "    callbacks.append(checkpoint_callback)\n",
    "\n",
    "    trainer = Trainer(\n",
    "                    devices = gpus,\n",
    "                    accelerator = \"cuda\" if len(gpus) >= 1 else 'cpu',\n",
    "                    max_epochs = max_epochs,\n",
    "                    num_sanity_val_steps = 2,\n",
    "                    callbacks = callbacks,\n",
    "    )\n",
    "\n",
    "    trainer.fit(model, pl_datamodule)\n",
    "\n",
    "    model = model.load_from_checkpoint(checkpoint_callback.best_model_path)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "hparams_range = {\n",
    "    \n",
    "    'predictor_hidden_dim' : ['suggest_int', ['predictor_hidden_dim', 16, 512]],\n",
    "    # 'predictor_output_dim' : ['suggest_int', ['emb_dim', 16, 512]],\n",
    "    \n",
    "    'p_m' : [\"suggest_float\", [\"p_m\", 0.1, 0.9]],\n",
    "    'alpha1' : [\"suggest_float\", [\"alpha1\", 0.1, 5]],\n",
    "    'alpha2' : [\"suggest_float\", [\"alpha2\", 0.1, 5]],\n",
    "    'beta' : [\"suggest_float\", [\"beta\", 0.1, 10]],\n",
    "    'K' : [\"suggest_int\", [\"K\", 2, 20]],\n",
    "\n",
    "\n",
    "    'lr' : ['suggest_float', ['lr', 0.0001, 0.05]],\n",
    "    'gamma' : ['suggest_float', ['gamma', 0.1, 0.95]],\n",
    "    'step_size' : ['suggest_int', ['step_size', 10, 100]],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import optuna\n",
    "import torch.nn.functional as F\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "def objective(      trial: optuna.trial.Trial,\n",
    "        ) -> float:\n",
    "        \"\"\"Objective function for optuna\n",
    "\n",
    "        Args:\n",
    "            trial: A object which returns hyperparameters of a model of hyperparameter search trial.\n",
    "            train_idx: Indices of training data in self.data and self.label.\n",
    "            test_idx: Indices of test data in self.data and self.label.\n",
    "            fold_idx: A fold index that denotes which fold under the given k-fold cross validation.\n",
    "        \n",
    "        Returns:\n",
    "            A score of given hyperparameters.\n",
    "        \"\"\"\n",
    "        model_hparams = {\n",
    "            \"encoder_dim\" : data.shape[1],\n",
    "            \"predictor_hidden_dim\" : None,\n",
    "            \"predictor_output_dim\" : 2,\n",
    "            'alpha1' : None,\n",
    "            'alpha2' : None,\n",
    "            'beta' : None,\n",
    "            'K' : None\n",
    "        }\n",
    "        \n",
    "        data_hparams = {\n",
    "            \"K\" : None,\n",
    "            \"p_m\" : None\n",
    "        }\n",
    "        optim_hparams = {\n",
    "            \"lr\" : None\n",
    "        }\n",
    "        scheduler_hparams = {\n",
    "            'gamma' : None,\n",
    "            'step_size' : None\n",
    "        }\n",
    "\n",
    "        for k, v in hparams_range.items():\n",
    "            if k in model_hparams.keys():\n",
    "                model_hparams[k] = getattr(trial, v[0])(*v[1])\n",
    "            if k in data_hparams.keys():\n",
    "                data_hparams[k] = getattr(trial, v[0])(*v[1])\n",
    "            if k in optim_hparams.keys():\n",
    "                optim_hparams[k] = getattr(trial, v[0])(*v[1])\n",
    "            if k in scheduler_hparams.keys():\n",
    "                scheduler_hparams[k] = getattr(trial, v[0])(*v[1])\n",
    "\n",
    "        \n",
    "        pl_vime = PLVIME(model_hparams, \"Adam\", optim_hparams, \"StepLR\", scheduler_hparams, \n",
    "        num_categoricals, num_continuous, -1, loss_fn,\n",
    "        AccuracyScorer(\"accuracy_score\"), random_seed)\n",
    "        \n",
    "        pl_vime = fit_model(pl_vime, data_hparams)\n",
    "        \n",
    "\n",
    "        trainer = Trainer(\n",
    "                    devices = gpus,\n",
    "                    accelerator = \"cuda\" if len(gpus)>= 1 else 'cpu',\n",
    "                    max_epochs = max_epochs,\n",
    "                    num_sanity_val_steps = 2,\n",
    "                    callbacks = None,\n",
    "        )\n",
    "        test_ds = VIMEClassificationDataset(X_valid, y_valid.values, data_hparams, None, continuous_cols, category_cols)\n",
    "        test_dl = DataLoader(test_ds, batch_size, shuffle=False, sampler = SequentialSampler(test_ds), num_workers=n_jobs)\n",
    "\n",
    "        preds = trainer.predict(pl_vime, test_dl)\n",
    "\n",
    "        preds = F.softmax(torch.concat([out.cpu() for out in preds]).squeeze(),dim=1)\n",
    "\n",
    "        accuracy = accuracy_score(y_valid, preds.argmax(1))\n",
    "\n",
    "        return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2023-12-18 16:33:37,870] A new study created in memory with name: no-name-c012dc04-28cf-40ce-9e1e-92be0cc74293\n",
      "Global seed set to 0\n",
      "/tmp/ipykernel_3969898/1358969741.py:20: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  train_ds = VIMESelfDataset(X_train.append(X_unlabeled), data_hparams, continuous_cols, category_cols)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "You are using a CUDA device ('NVIDIA GeForce RTX 4090') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data/pretraining exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 150 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "150 K     Trainable params\n",
      "0         Non-trainable params\n",
      "150 K     Total params\n",
      "0.602     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: 100%|██████████| 67/67 [00:04<00:00, 13.99it/s, v_num=384, train_loss=3.760, val_loss=3.760]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=30` reached.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: 100%|██████████| 67/67 [00:04<00:00, 13.99it/s, v_num=384, train_loss=3.760, val_loss=3.760]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "/workspace/vime/data_utils.py:57: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  X = X.append(unlabeled_data)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 150 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "150 K     Trainable params\n",
      "0         Non-trainable params\n",
      "150 K     Total params\n",
      "0.602     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: 100%|██████████| 67/67 [00:11<00:00,  5.59it/s, v_num=385, train_loss=0.626, train_accuracy_score=0.812, val_accuracy_score=0.836, val_loss=0.602]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=30` reached.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: 100%|██████████| 67/67 [00:11<00:00,  5.59it/s, v_num=385, train_loss=0.626, train_accuracy_score=0.812, val_accuracy_score=0.836, val_loss=0.602]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 29/29 [00:00<00:00, 666.54it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2023-12-18 16:42:35,212] Trial 0 finished with value: 0.8394185491025729 and parameters: {'predictor_hidden_dim': 288, 'p_m': 0.6721514930979355, 'alpha1': 3.053540542751055, 'alpha2': 2.769927596684795, 'beta': 4.294182513455157, 'K': 14, 'lr': 0.021935601842008354, 'gamma': 0.8580070506647678, 'step_size': 97}. Best is trial 0 with value: 0.8394185491025729.\n",
      "Global seed set to 0\n",
      "/tmp/ipykernel_3969898/1358969741.py:20: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  train_ds = VIMESelfDataset(X_train.append(X_unlabeled), data_hparams, continuous_cols, category_cols)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data/pretraining exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 100 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "100 K     Trainable params\n",
      "0         Non-trainable params\n",
      "100 K     Total params\n",
      "0.403     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16: 100%|██████████| 67/67 [00:05<00:00, 13.28it/s, v_num=387, train_loss=3.400, val_loss=3.400]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "/workspace/vime/data_utils.py:57: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  X = X.append(unlabeled_data)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 100 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "100 K     Trainable params\n",
      "0         Non-trainable params\n",
      "100 K     Total params\n",
      "0.403     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10: 100%|██████████| 67/67 [00:05<00:00, 12.90it/s, v_num=388, train_loss=0.673, train_accuracy_score=0.824, val_accuracy_score=0.814, val_loss=0.661]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 29/29 [00:00<00:00, 910.76it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2023-12-18 16:45:09,889] Trial 1 finished with value: 0.8334811983894083 and parameters: {'predictor_hidden_dim': 206, 'p_m': 0.7333800304661316, 'alpha1': 2.691585106789232, 'alpha2': 2.8834183493602685, 'beta': 9.263406719097345, 'K': 3, 'lr': 0.004447752055106882, 'gamma': 0.11718563782427686, 'step_size': 85}. Best is trial 0 with value: 0.8394185491025729.\n",
      "Global seed set to 0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3969898/1358969741.py:20: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  train_ds = VIMESelfDataset(X_train.append(X_unlabeled), data_hparams, continuous_cols, category_cols)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data/pretraining exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 241 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "241 K     Trainable params\n",
      "0         Non-trainable params\n",
      "241 K     Total params\n",
      "0.968     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 22: 100%|██████████| 67/67 [00:04<00:00, 14.72it/s, v_num=390, train_loss=5.990, val_loss=6.000]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "/workspace/vime/data_utils.py:57: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  X = X.append(unlabeled_data)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 241 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "241 K     Trainable params\n",
      "0         Non-trainable params\n",
      "241 K     Total params\n",
      "0.968     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11: 100%|██████████| 67/67 [00:12<00:00,  5.28it/s, v_num=391, train_loss=0.662, train_accuracy_score=0.819, val_accuracy_score=0.818, val_loss=0.641]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 29/29 [00:00<00:00, 909.43it/s] "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2023-12-18 16:50:13,419] Trial 2 finished with value: 0.8344366341363544 and parameters: {'predictor_hidden_dim': 402, 'p_m': 0.7960097185974554, 'alpha1': 4.895229876940544, 'alpha2': 4.015876964661945, 'beta': 4.668645686304025, 'K': 16, 'lr': 0.0060018938508597675, 'gamma': 0.6439328681283952, 'step_size': 23}. Best is trial 0 with value: 0.8394185491025729.\n",
      "Global seed set to 0\n",
      "/tmp/ipykernel_3969898/1358969741.py:20: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  train_ds = VIMESelfDataset(X_train.append(X_unlabeled), data_hparams, continuous_cols, category_cols)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data/pretraining exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 324 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "324 K     Trainable params\n",
      "0         Non-trainable params\n",
      "324 K     Total params\n",
      "1.299     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 14: 100%|██████████| 67/67 [00:05<00:00, 12.05it/s, v_num=393, train_loss=2.650, val_loss=2.650]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "/workspace/vime/data_utils.py:57: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  X = X.append(unlabeled_data)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 324 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "324 K     Trainable params\n",
      "0         Non-trainable params\n",
      "324 K     Total params\n",
      "1.299     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11: 100%|██████████| 67/67 [00:09<00:00,  6.79it/s, v_num=394, train_loss=0.671, train_accuracy_score=0.807, val_accuracy_score=0.788, val_loss=0.634]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 29/29 [00:00<00:00, 675.21it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2023-12-18 16:53:49,137] Trial 3 finished with value: 0.824336313382925 and parameters: {'predictor_hidden_dim': 485, 'p_m': 0.5174786574000574, 'alpha1': 2.1318435059535656, 'alpha2': 1.3963224993126724, 'beta': 7.764913525398745, 'K': 10, 'lr': 0.02846485404854556, 'gamma': 0.11597133037090188, 'step_size': 66}. Best is trial 0 with value: 0.8394185491025729.\n",
      "Global seed set to 0\n",
      "/tmp/ipykernel_3969898/1358969741.py:20: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  train_ds = VIMESelfDataset(X_train.append(X_unlabeled), data_hparams, continuous_cols, category_cols)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data/pretraining exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 173 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "173 K     Trainable params\n",
      "0         Non-trainable params\n",
      "173 K     Total params\n",
      "0.694     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 20: 100%|██████████| 67/67 [00:04<00:00, 13.46it/s, v_num=396, train_loss=5.730, val_loss=5.730]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "/workspace/vime/data_utils.py:57: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  X = X.append(unlabeled_data)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 173 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "173 K     Trainable params\n",
      "0         Non-trainable params\n",
      "173 K     Total params\n",
      "0.694     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12: 100%|██████████| 67/67 [00:09<00:00,  7.16it/s, v_num=397, train_loss=0.678, train_accuracy_score=0.795, val_accuracy_score=0.746, val_loss=0.617]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 29/29 [00:00<00:00, 1091.21it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2023-12-18 16:57:53,123] Trial 4 finished with value: 0.8014741008667167 and parameters: {'predictor_hidden_dim': 320, 'p_m': 0.5935471974998056, 'alpha1': 4.724365584721658, 'alpha2': 3.440919465607069, 'beta': 3.659128215680482, 'K': 10, 'lr': 0.034911796676770517, 'gamma': 0.15119165088487935, 'step_size': 70}. Best is trial 0 with value: 0.8394185491025729.\n",
      "Global seed set to 0\n",
      "/tmp/ipykernel_3969898/1358969741.py:20: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  train_ds = VIMESelfDataset(X_train.append(X_unlabeled), data_hparams, continuous_cols, category_cols)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data/pretraining exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 196 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "196 K     Trainable params\n",
      "0         Non-trainable params\n",
      "196 K     Total params\n",
      "0.785     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 29: 100%|██████████| 67/67 [00:04<00:00, 13.77it/s, v_num=399, train_loss=0.884, val_loss=0.885]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=30` reached.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: 100%|██████████| 67/67 [00:04<00:00, 13.76it/s, v_num=399, train_loss=0.884, val_loss=0.885]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "/workspace/vime/data_utils.py:57: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  X = X.append(unlabeled_data)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 196 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "196 K     Trainable params\n",
      "0         Non-trainable params\n",
      "196 K     Total params\n",
      "0.785     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10: 100%|██████████| 67/67 [00:11<00:00,  6.02it/s, v_num=400, train_loss=0.542, train_accuracy_score=0.847, val_accuracy_score=0.787, val_loss=0.490]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 29/29 [00:00<00:00, 640.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2023-12-18 17:02:43,184] Trial 5 finished with value: 0.8165563365863646 and parameters: {'predictor_hidden_dim': 349, 'p_m': 0.2683060488590727, 'alpha1': 0.7317388585087812, 'alpha2': 1.6455989195285012, 'beta': 3.700736632331964, 'K': 12, 'lr': 0.021986215521769784, 'gamma': 0.9401177623503422, 'step_size': 19}. Best is trial 0 with value: 0.8394185491025729.\n",
      "Global seed set to 0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3969898/1358969741.py:20: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  train_ds = VIMESelfDataset(X_train.append(X_unlabeled), data_hparams, continuous_cols, category_cols)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data/pretraining exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 62.8 K\n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "62.8 K    Trainable params\n",
      "0         Non-trainable params\n",
      "62.8 K    Total params\n",
      "0.251     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 21: 100%|██████████| 67/67 [00:04<00:00, 13.67it/s, v_num=402, train_loss=3.780, val_loss=3.780]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "/workspace/vime/data_utils.py:57: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  X = X.append(unlabeled_data)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 62.8 K\n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "62.8 K    Trainable params\n",
      "0         Non-trainable params\n",
      "62.8 K    Total params\n",
      "0.251     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10: 100%|██████████| 67/67 [00:06<00:00,  9.83it/s, v_num=403, train_loss=0.566, train_accuracy_score=0.819, val_accuracy_score=0.781, val_loss=0.519]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 29/29 [00:00<00:00, 704.04it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2023-12-18 17:06:15,486] Trial 6 finished with value: 0.8060465433699584 and parameters: {'predictor_hidden_dim': 119, 'p_m': 0.22904761430799703, 'alpha1': 3.3002307947804526, 'alpha2': 1.3411288524449325, 'beta': 4.716476651277432, 'K': 6, 'lr': 0.008032582223911433, 'gamma': 0.19381886998965936, 'step_size': 69}. Best is trial 0 with value: 0.8394185491025729.\n",
      "Global seed set to 0\n",
      "/tmp/ipykernel_3969898/1358969741.py:20: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  train_ds = VIMESelfDataset(X_train.append(X_unlabeled), data_hparams, continuous_cols, category_cols)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data/pretraining exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 51.8 K\n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "51.8 K    Trainable params\n",
      "0         Non-trainable params\n",
      "51.8 K    Total params\n",
      "0.207     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 29: 100%|██████████| 67/67 [00:04<00:00, 13.41it/s, v_num=405, train_loss=2.170, val_loss=2.170]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=30` reached.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: 100%|██████████| 67/67 [00:05<00:00, 13.40it/s, v_num=405, train_loss=2.170, val_loss=2.170]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "/workspace/vime/data_utils.py:57: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  X = X.append(unlabeled_data)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 51.8 K\n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "51.8 K    Trainable params\n",
      "0         Non-trainable params\n",
      "51.8 K    Total params\n",
      "0.207     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14: 100%|██████████| 67/67 [00:13<00:00,  4.84it/s, v_num=406, train_loss=0.389, train_accuracy_score=0.899, val_accuracy_score=0.815, val_loss=0.405]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 29/29 [00:00<00:00, 565.88it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2023-12-18 17:12:40,726] Trial 7 finished with value: 0.8186036989012488 and parameters: {'predictor_hidden_dim': 84, 'p_m': 0.2572658893440428, 'alpha1': 1.9067533362387243, 'alpha2': 4.1228668262548815, 'beta': 1.0613026303513067, 'K': 17, 'lr': 0.004895310553908757, 'gamma': 0.9299905452613864, 'step_size': 52}. Best is trial 0 with value: 0.8394185491025729.\n",
      "Global seed set to 0\n",
      "/tmp/ipykernel_3969898/1358969741.py:20: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  train_ds = VIMESelfDataset(X_train.append(X_unlabeled), data_hparams, continuous_cols, category_cols)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data/pretraining exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 342 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "342 K     Trainable params\n",
      "0         Non-trainable params\n",
      "342 K     Total params\n",
      "1.370     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 14: 100%|██████████| 67/67 [00:05<00:00, 12.44it/s, v_num=408, train_loss=4.470, val_loss=4.470]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "/workspace/vime/data_utils.py:57: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  X = X.append(unlabeled_data)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 342 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "342 K     Trainable params\n",
      "0         Non-trainable params\n",
      "342 K     Total params\n",
      "1.370     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11: 100%|██████████| 67/67 [00:05<00:00, 12.51it/s, v_num=409, train_loss=0.628, train_accuracy_score=0.826, val_accuracy_score=0.798, val_loss=0.593]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 29/29 [00:00<00:00, 1007.06it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2023-12-18 17:15:26,074] Trial 8 finished with value: 0.8104824950522077 and parameters: {'predictor_hidden_dim': 501, 'p_m': 0.5838764157960368, 'alpha1': 3.7223915390516784, 'alpha2': 0.2920201820461713, 'beta': 2.899788929506455, 'K': 4, 'lr': 0.014877395856355032, 'gamma': 0.20091856111110745, 'step_size': 38}. Best is trial 0 with value: 0.8394185491025729.\n",
      "Global seed set to 0\n",
      "/tmp/ipykernel_3969898/1358969741.py:20: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  train_ds = VIMESelfDataset(X_train.append(X_unlabeled), data_hparams, continuous_cols, category_cols)\n",
      "GPU available: True (cuda), used: True\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data/pretraining exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 108 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "108 K     Trainable params\n",
      "0         Non-trainable params\n",
      "108 K     Total params\n",
      "0.436     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18: 100%|██████████| 67/67 [00:04<00:00, 13.65it/s, v_num=411, train_loss=3.780, val_loss=3.790]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "/workspace/vime/data_utils.py:57: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  X = X.append(unlabeled_data)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 108 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "108 K     Trainable params\n",
      "0         Non-trainable params\n",
      "108 K     Total params\n",
      "0.436     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17: 100%|██████████| 67/67 [00:09<00:00,  6.84it/s, v_num=412, train_loss=0.447, train_accuracy_score=0.854, val_accuracy_score=0.787, val_loss=0.433]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 29/29 [00:00<00:00, 650.81it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2023-12-18 17:20:31,636] Trial 9 finished with value: 0.8075479424008736 and parameters: {'predictor_hidden_dim': 221, 'p_m': 0.1513179970790275, 'alpha1': 3.4931133849130975, 'alpha2': 2.8763471256122184, 'beta': 2.72735596030051, 'K': 11, 'lr': 0.00478763148684624, 'gamma': 0.5895545212227524, 'step_size': 94}. Best is trial 0 with value: 0.8394185491025729.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Number of finished trials:  10\n",
      "Best trial:\n",
      "  Accuracy: 0.8394185491025729\n",
      "  Best hyperparameters:  FrozenTrial(number=0, state=TrialState.COMPLETE, values=[0.8394185491025729], datetime_start=datetime.datetime(2023, 12, 18, 16, 33, 37, 871513), datetime_complete=datetime.datetime(2023, 12, 18, 16, 42, 35, 212155), params={'predictor_hidden_dim': 288, 'p_m': 0.6721514930979355, 'alpha1': 3.053540542751055, 'alpha2': 2.769927596684795, 'beta': 4.294182513455157, 'K': 14, 'lr': 0.021935601842008354, 'gamma': 0.8580070506647678, 'step_size': 97}, user_attrs={}, system_attrs={}, intermediate_values={}, distributions={'predictor_hidden_dim': IntDistribution(high=512, log=False, low=16, step=1), 'p_m': FloatDistribution(high=0.9, log=False, low=0.1, step=None), 'alpha1': FloatDistribution(high=5.0, log=False, low=0.1, step=None), 'alpha2': FloatDistribution(high=5.0, log=False, low=0.1, step=None), 'beta': FloatDistribution(high=10.0, log=False, low=0.1, step=None), 'K': IntDistribution(high=20, log=False, low=2, step=1), 'lr': FloatDistribution(high=0.05, log=False, low=0.0001, step=None), 'gamma': FloatDistribution(high=0.95, log=False, low=0.1, step=None), 'step_size': IntDistribution(high=100, log=False, low=10, step=1)}, trial_id=0, value=None)\n"
     ]
    }
   ],
   "source": [
    "study = optuna.create_study(direction=\"maximize\",sampler=optuna.samplers.TPESampler(seed=random_seed))\n",
    "study.optimize(objective, n_trials=10, show_progress_bar=False)\n",
    "\n",
    "print(\"Number of finished trials: \", len(study.trials))\n",
    "print(\"Best trial:\")\n",
    "\n",
    "\n",
    "trial = study.best_trial\n",
    "\n",
    "print(\"  Accuracy: {}\".format(trial.value))\n",
    "print(\"  Best hyperparameters: \", trial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "/tmp/ipykernel_3969898/1358969741.py:20: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  train_ds = VIMESelfDataset(X_train.append(X_unlabeled), data_hparams, continuous_cols, category_cols)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data/pretraining exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 150 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "150 K     Trainable params\n",
      "0         Non-trainable params\n",
      "150 K     Total params\n",
      "0.602     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: 100%|██████████| 67/67 [00:04<00:00, 13.58it/s, v_num=414, train_loss=3.760, val_loss=3.760]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=30` reached.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: 100%|██████████| 67/67 [00:04<00:00, 13.58it/s, v_num=414, train_loss=3.760, val_loss=3.760]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n",
      "/workspace/vime/data_utils.py:57: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  X = X.append(unlabeled_data)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/opt/conda/envs/py38/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /workspace/vime/temporary_ckpt_data exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name                      | Type             | Params\n",
      "---------------------------------------------------------------\n",
      "0 | model                     | VIME             | 150 K \n",
      "1 | pretraining_mask_loss     | BCELoss          | 0     \n",
      "2 | pretraining_feature_loss1 | CrossEntropyLoss | 0     \n",
      "3 | pretraining_feature_loss2 | MSELoss          | 0     \n",
      "4 | consistency_loss          | MSELoss          | 0     \n",
      "5 | loss_fn                   | CrossEntropyLoss | 0     \n",
      "---------------------------------------------------------------\n",
      "150 K     Trainable params\n",
      "0         Non-trainable params\n",
      "150 K     Total params\n",
      "0.602     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: 100%|██████████| 67/67 [00:12<00:00,  5.41it/s, v_num=415, train_loss=0.626, train_accuracy_score=0.812, val_accuracy_score=0.836, val_loss=0.602]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=30` reached.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29: 100%|██████████| 67/67 [00:12<00:00,  5.41it/s, v_num=415, train_loss=0.626, train_accuracy_score=0.812, val_accuracy_score=0.836, val_loss=0.602]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 0\n"
     ]
    }
   ],
   "source": [
    "model_hparams = {\n",
    "            \"encoder_dim\" : data.shape[1],\n",
    "            \"predictor_hidden_dim\" : None,\n",
    "            \"predictor_output_dim\" : 2,\n",
    "            'alpha1' : None,\n",
    "            'alpha2' : None,\n",
    "            'beta' : None,\n",
    "            'K' : None\n",
    "        }\n",
    "        \n",
    "data_hparams = {\n",
    "        \"K\" : None,\n",
    "        \"p_m\" : None\n",
    "}\n",
    "optim_hparams = {\n",
    "        \"lr\" : None\n",
    "}\n",
    "scheduler_hparams = {\n",
    "        'gamma' : None,\n",
    "        'step_size' : None\n",
    "}\n",
    "\n",
    "for k, v in study.best_trial.params.items():\n",
    "        if k in model_hparams.keys():\n",
    "                model_hparams[k] = study.best_trial.params[k]\n",
    "        if k in data_hparams.keys():\n",
    "                data_hparams[k] = study.best_trial.params[k]\n",
    "        if k in optim_hparams.keys():\n",
    "                optim_hparams[k] = study.best_trial.params[k]\n",
    "        if k in scheduler_hparams.keys():\n",
    "                scheduler_hparams[k] = study.best_trial.params[k]\n",
    "\n",
    "\n",
    "pl_vime = PLVIME(model_hparams, \"Adam\", optim_hparams, \"StepLR\", scheduler_hparams, \n",
    "        num_categoricals, num_continuous, -1, loss_fn,\n",
    "        AccuracyScorer(\"accuracy_score\"), random_seed)\n",
    "pl_vime = fit_model(pl_vime, data_hparams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 29/29 [00:00<00:00, 668.86it/s]\n"
     ]
    }
   ],
   "source": [
    "import torch.nn.functional as F\n",
    "\n",
    "trainer = Trainer(\n",
    "                    devices = gpus,\n",
    "                    accelerator = \"cuda\" if len(gpus)>= 1 else 'cpu',\n",
    "                    max_epochs = max_epochs,\n",
    "                    num_sanity_val_steps = 2,\n",
    "                    callbacks = None,\n",
    "    )\n",
    "test_ds = VIMEClassificationDataset(X_valid, y_valid.values, data_hparams, None, continuous_cols, category_cols)\n",
    "test_dl = DataLoader(test_ds, batch_size, shuffle=False, sampler = SequentialSampler(test_ds), num_workers=n_jobs)\n",
    "\n",
    "preds = trainer.predict(pl_vime, test_dl)\n",
    "\n",
    "preds = F.softmax(torch.concat([out.cpu() for out in preds]).squeeze(),dim=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8394185491025729"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "accuracy_score(y_valid, preds.argmax(1))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py37",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
